<!DOCTYPE html>

<html lang="en">

<head>
    <meta charset="utf-8">
    <meta http-equiv="X-UA-Compatible" content="IE=edge">
    <meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=no">
    <meta name="apple-mobile-web-app-capable" content="yes">
    <meta name="apple-mobile-web-app-status-bar-style" content="black">
    <meta name="mobile-web-app-capable" content="yes">
    <title>
        Implementation of ResNet - HackMD
    </title>
    <link rel="icon" type="image/png" href="https://hackmd.io/favicon.png">
    <link rel="apple-touch-icon" href="https://hackmd.io/apple-touch-icon.png">

    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.3.7/css/bootstrap.min.css" integrity="sha256-916EbMg70RQy9LHiGkXzG8hSg9EdNy97GazNG/aiY1w=" crossorigin="anonymous" />
    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css" integrity="sha256-eZrrJcwDc/3uDhsdt61sL2oOBY362qM3lon1gyExkL0=" crossorigin="anonymous" />
    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/ionicons/2.0.1/css/ionicons.min.css" integrity="sha256-3iu9jgsy9TpTwXKb7bNQzqWekRX7pPK+2OLj3R922fo=" crossorigin="anonymous" />
    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/octicons/3.5.0/octicons.min.css" integrity="sha256-QiWfLIsCT02Sdwkogf6YMiQlj4NE84MKkzEMkZnMGdg=" crossorigin="anonymous" />
    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/prism/1.5.1/themes/prism.min.css" integrity="sha256-vtR0hSWRc3Tb26iuN2oZHt3KRUomwTufNIf5/4oeCyg=" crossorigin="anonymous" />
    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@hackmd/emojify.js@2.1.0/dist/css/basic/emojify.min.css" integrity="sha256-UOrvMOsSDSrW6szVLe8ZDZezBxh5IoIfgTwdNDgTjiU=" crossorigin="anonymous" />
    <style>
        @charset "UTF-8";@import url(https://fonts.googleapis.com/css?family=Roboto:300,300i,400,400i,500,500i|Source+Code+Pro:300,400,500|Source+Sans+Pro:300,300i,400,400i,600,600i|Source+Serif+Pro&subset=latin-ext);.hljs{display:block;background:#fff;padding:.5em;color:#333;overflow-x:auto}.hljs-comment,.hljs-meta{color:#969896}.hljs-emphasis,.hljs-quote,.hljs-string,.hljs-strong,.hljs-template-variable,.hljs-variable{color:#df5000}.hljs-keyword,.hljs-selector-tag,.hljs-type{color:#a71d5d}.hljs-attribute,.hljs-bullet,.hljs-literal,.hljs-number,.hljs-symbol{color:#0086b3}.hljs-built_in,.hljs-builtin-name{color:#005cc5}.hljs-name,.hljs-section{color:#63a35c}.hljs-tag{color:#333}.hljs-attr,.hljs-selector-attr,.hljs-selector-class,.hljs-selector-id,.hljs-selector-pseudo,.hljs-title{color:#795da3}.hljs-addition{color:#55a532;background-color:#eaffea}.hljs-deletion{color:#bd2c00;background-color:#ffecec}.hljs-link{text-decoration:underline}.markdown-body{font-size:16px;line-height:1.5;word-wrap:break-word}.markdown-body:after,.markdown-body:before{display:table;content:""}.markdown-body:after{clear:both}.markdown-body>:first-child{margin-top:0!important}.markdown-body>:last-child{margin-bottom:0!important}.markdown-body a:not([href]){color:inherit;text-decoration:none}.markdown-body .absent{color:#c00}.markdown-body .anchor{float:left;padding-right:4px;margin-left:-20px;line-height:1}.markdown-body .anchor:focus{outline:none}.markdown-body blockquote,.markdown-body dl,.markdown-body ol,.markdown-body p,.markdown-body pre,.markdown-body table,.markdown-body ul{margin-top:0;margin-bottom:16px}.markdown-body hr{height:.25em;padding:0;margin:24px 0;background-color:#e7e7e7;border:0}.markdown-body blockquote{font-size:16px;padding:0 1em;color:#777;border-left:.25em solid #ddd}.markdown-body blockquote>:first-child{margin-top:0}.markdown-body blockquote>:last-child{margin-bottom:0}.markdown-body kbd,.popover kbd{display:inline-block;padding:3px 5px;font-size:11px;line-height:10px;color:#555;vertical-align:middle;background-color:#fcfcfc;border:1px solid #ccc;border-bottom-color:#bbb;border-radius:3px;box-shadow:inset 0 -1px 0 #bbb}.markdown-body .loweralpha{list-style-type:lower-alpha}.markdown-body h1,.markdown-body h2,.markdown-body h3,.markdown-body h4,.markdown-body h5,.markdown-body h6{margin-top:24px;margin-bottom:16px;font-weight:600;line-height:1.25}.markdown-body h1 .octicon-link,.markdown-body h2 .octicon-link,.markdown-body h3 .octicon-link,.markdown-body h4 .octicon-link,.markdown-body h5 .octicon-link,.markdown-body h6 .octicon-link{color:#000;vertical-align:middle;visibility:hidden}.markdown-body h1:hover .anchor,.markdown-body h2:hover .anchor,.markdown-body h3:hover .anchor,.markdown-body h4:hover .anchor,.markdown-body h5:hover .anchor,.markdown-body h6:hover .anchor{text-decoration:none}.markdown-body h1:hover .anchor .octicon-link,.markdown-body h2:hover .anchor .octicon-link,.markdown-body h3:hover .anchor .octicon-link,.markdown-body h4:hover .anchor .octicon-link,.markdown-body h5:hover .anchor .octicon-link,.markdown-body h6:hover .anchor .octicon-link{visibility:visible}.markdown-body h1 code,.markdown-body h1 tt,.markdown-body h2 code,.markdown-body h2 tt,.markdown-body h3 code,.markdown-body h3 tt,.markdown-body h4 code,.markdown-body h4 tt,.markdown-body h5 code,.markdown-body h5 tt,.markdown-body h6 code,.markdown-body h6 tt{font-size:inherit}.markdown-body h1{font-size:2em}.markdown-body h1,.markdown-body h2{padding-bottom:.3em;border-bottom:1px solid #eee}.markdown-body h2{font-size:1.5em}.markdown-body h3{font-size:1.25em}.markdown-body h4{font-size:1em}.markdown-body h5{font-size:.875em}.markdown-body h6{font-size:.85em;color:#777}.markdown-body ol,.markdown-body ul{padding-left:2em}.markdown-body ol.no-list,.markdown-body ul.no-list{padding:0;list-style-type:none}.markdown-body ol ol,.markdown-body ol ul,.markdown-body ul ol,.markdown-body ul ul{margin-top:0;margin-bottom:0}.markdown-body li>p{margin-top:16px}.markdown-body li+li{padding-top:.25em}.markdown-body dl{padding:0}.markdown-body dl dt{padding:0;margin-top:16px;font-size:1em;font-style:italic;font-weight:700}.markdown-body dl dd{padding:0 16px;margin-bottom:16px}.markdown-body table{display:block;width:100%;overflow:auto;word-break:normal;word-break:keep-all}.markdown-body table th{font-weight:700}.markdown-body table td,.markdown-body table th{padding:6px 13px;border:1px solid #ddd}.markdown-body table tr{background-color:#fff;border-top:1px solid #ccc}.markdown-body table tr:nth-child(2n){background-color:#f8f8f8}.markdown-body img{max-width:100%;box-sizing:content-box;background-color:#fff}.markdown-body img[align=right]{padding-left:20px}.markdown-body img[align=left]{padding-right:20px}.markdown-body .emoji{max-width:none;vertical-align:text-top;background-color:transparent}.markdown-body span.frame{display:block;overflow:hidden}.markdown-body span.frame>span{display:block;float:left;width:auto;padding:7px;margin:13px 0 0;overflow:hidden;border:1px solid #ddd}.markdown-body span.frame span img{display:block;float:left}.markdown-body span.frame span span{display:block;padding:5px 0 0;clear:both;color:#333}.markdown-body span.align-center{display:block;overflow:hidden;clear:both}.markdown-body span.align-center>span{display:block;margin:13px auto 0;overflow:hidden;text-align:center}.markdown-body span.align-center span img{margin:0 auto;text-align:center}.markdown-body span.align-right{display:block;overflow:hidden;clear:both}.markdown-body span.align-right>span{display:block;margin:13px 0 0;overflow:hidden;text-align:right}.markdown-body span.align-right span img{margin:0;text-align:right}.markdown-body span.float-left{display:block;float:left;margin-right:13px;overflow:hidden}.markdown-body span.float-left span{margin:13px 0 0}.markdown-body span.float-right{display:block;float:right;margin-left:13px;overflow:hidden}.markdown-body span.float-right>span{display:block;margin:13px auto 0;overflow:hidden;text-align:right}.markdown-body code,.markdown-body tt{padding:0;padding-top:.2em;padding-bottom:.2em;margin:0;font-size:85%;background-color:rgba(0,0,0,.04);border-radius:3px}.markdown-body code:after,.markdown-body code:before,.markdown-body tt:after,.markdown-body tt:before{letter-spacing:-.2em;content:"\00a0"}.markdown-body code br,.markdown-body tt br{display:none}.markdown-body del code{text-decoration:inherit}.markdown-body pre{word-wrap:normal}.markdown-body pre>code{padding:0;margin:0;font-size:100%;word-break:normal;white-space:pre;background:transparent;border:0}.markdown-body .highlight{margin-bottom:16px}.markdown-body .highlight pre{margin-bottom:0;word-break:normal}.markdown-body .highlight pre,.markdown-body pre{padding:16px;overflow:auto;font-size:85%;line-height:1.45;background-color:#f7f7f7;border-radius:3px}.markdown-body pre code,.markdown-body pre tt{display:inline;max-width:auto;padding:0;margin:0;overflow:visible;line-height:inherit;word-wrap:normal;background-color:transparent;border:0}.markdown-body pre code:after,.markdown-body pre code:before,.markdown-body pre tt:after,.markdown-body pre tt:before{content:normal}.markdown-body .csv-data td,.markdown-body .csv-data th{padding:5px;overflow:hidden;font-size:12px;line-height:1;text-align:left;white-space:nowrap}.markdown-body .csv-data .blob-line-num{padding:10px 8px 9px;text-align:right;background:#fff;border:0}.markdown-body .csv-data tr{border-top:0}.markdown-body .csv-data th{font-weight:700;background:#f8f8f8;border-top:0}.news .alert .markdown-body blockquote{padding:0 0 0 40px;border:0 none}.activity-tab .news .alert .commits,.activity-tab .news .markdown-body blockquote{padding-left:0}.task-list-item{list-style-type:none}.task-list-item label{font-weight:400}.task-list-item.enabled label{cursor:pointer}.task-list-item+.task-list-item{margin-top:3px}.task-list-item-checkbox{float:left;margin:.31em 0 .2em -1.3em!important;vertical-align:middle;cursor:default!important}.markdown-body{padding-top:40px;padding-bottom:40px;max-width:758px;overflow:visible!important}.markdown-body .emoji{vertical-align:top}.markdown-body pre{border:inherit!important}.markdown-body code{color:inherit!important}.markdown-body pre code .wrapper{display:-moz-inline-flex;display:-ms-inline-flex;display:-o-inline-flex;display:inline-flex}.markdown-body pre code .gutter{float:left;overflow:hidden;-webkit-user-select:none;user-select:none}.markdown-body pre code .gutter.linenumber{text-align:right;position:relative;display:inline-block;cursor:default;z-index:4;padding:0 8px 0 0;min-width:20px;box-sizing:content-box;color:#afafaf!important;border-right:3px solid #6ce26c!important}.markdown-body pre code .gutter.linenumber>span:before{content:attr(data-linenumber)}.markdown-body pre code .code{float:left;margin:0 0 0 16px}.markdown-body .gist .line-numbers{border-left:none;border-top:none;border-bottom:none}.markdown-body .gist .line-data{border:none}.markdown-body .gist table{border-spacing:0;border-collapse:inherit!important}.markdown-body code[data-gist-id]{background:none;padding:0}.markdown-body code[data-gist-id]:after,.markdown-body code[data-gist-id]:before{content:""}.markdown-body code[data-gist-id] .blob-num{border:unset}.markdown-body code[data-gist-id] table{overflow:unset;margin-bottom:unset}.markdown-body code[data-gist-id] table tr{background:unset}.markdown-body[dir=rtl] pre{direction:ltr}.markdown-body[dir=rtl] code{direction:ltr;unicode-bidi:embed}.markdown-body .alert>p{margin-bottom:0}.markdown-body pre.abc,.markdown-body pre.flow-chart,.markdown-body pre.graphviz,.markdown-body pre.mermaid,.markdown-body pre.sequence-diagram,.markdown-body pre.vega{text-align:center;background-color:inherit;border-radius:0;white-space:inherit}.markdown-body pre.abc>code,.markdown-body pre.flow-chart>code,.markdown-body pre.graphviz>code,.markdown-body pre.mermaid>code,.markdown-body pre.sequence-diagram>code,.markdown-body pre.vega>code{text-align:left}.markdown-body pre.abc>svg,.markdown-body pre.flow-chart>svg,.markdown-body pre.graphviz>svg,.markdown-body pre.mermaid>svg,.markdown-body pre.sequence-diagram>svg,.markdown-body pre.vega>svg{max-width:100%;height:100%}.markdown-body pre>code.wrap{white-space:pre-wrap;white-space:-moz-pre-wrap;white-space:-pre-wrap;white-space:-o-pre-wrap;word-wrap:break-word}.markdown-body .alert>p,.markdown-body .alert>ul{margin-bottom:0}.markdown-body summary{display:list-item}.markdown-body summary:focus{outline:none}.markdown-body details summary{cursor:pointer}.markdown-body details:not([open])>:not(summary){display:none}.markdown-body figure{margin:1em 40px}.markdown-body .mark,.markdown-body mark{background-color:#fff1a7}.vimeo,.youtube{cursor:pointer;display:table;text-align:center;background-position:50%;background-repeat:no-repeat;background-size:contain;background-color:#000;overflow:hidden}.vimeo,.youtube{position:relative;width:100%}.youtube{padding-bottom:56.25%}.vimeo img{width:100%;object-fit:contain;z-index:0}.youtube img{object-fit:cover;z-index:0}.vimeo iframe,.youtube iframe,.youtube img{width:100%;height:100%;position:absolute;top:0;left:0}.vimeo iframe,.youtube iframe{vertical-align:middle;z-index:1}.vimeo .icon,.youtube .icon{position:absolute;height:auto;width:auto;top:50%;left:50%;transform:translate(-50%,-50%);color:#fff;opacity:.3;transition:opacity .2s;z-index:0}.vimeo:hover .icon,.youtube:hover .icon{opacity:.6;transition:opacity .2s}.slideshare .inner,.speakerdeck .inner{position:relative;width:100%}.slideshare .inner iframe,.speakerdeck .inner iframe{position:absolute;top:0;bottom:0;left:0;right:0;width:100%;height:100%}.MJX_Assistive_MathML{display:none}.ui-infobar{position:relative;z-index:2;max-width:760px;margin:25px auto -25px;padding:0 15px;color:#777}.toc .invisable-node{list-style-type:none}.ui-toc{position:fixed;bottom:20px;z-index:998}.ui-toc-label{opacity:.3;background-color:#ccc;border:none;transition:opacity .2s}.ui-toc .open .ui-toc-label{opacity:1;color:#fff;transition:opacity .2s}.ui-toc-label:focus{opacity:.3;background-color:#ccc;color:#000}.ui-toc-label:hover{opacity:1;background-color:#ccc;transition:opacity .2s}.ui-toc-dropdown{margin-top:23px;margin-bottom:20px;padding-left:10px;padding-right:10px;max-width:45vw;width:25vw;max-height:70vh;overflow:auto;text-align:inherit}.ui-toc-dropdown>.toc{max-height:calc(70vh - 100px);overflow:auto}.ui-toc-dropdown[dir=rtl] .nav{padding-right:0;letter-spacing:.0029em}.ui-toc-dropdown a{overflow:hidden;text-overflow:ellipsis;white-space:pre}.ui-toc-dropdown .nav>li>a{display:block;padding:4px 20px;font-size:13px;font-weight:500;color:#767676}.ui-toc-dropdown .nav>li:first-child:last-child > ul,.ui-toc-dropdown .toc.expand ul{display:block}.ui-toc-dropdown .nav>li>a:focus,.ui-toc-dropdown .nav>li>a:hover{padding-left:19px;color:#000;text-decoration:none;background-color:transparent;border-left:1px solid #000}.ui-toc-dropdown[dir=rtl] .nav>li>a:focus,.ui-toc-dropdown[dir=rtl] .nav>li>a:hover{padding-right:19px;border-left:none;border-right:1px solid #000}.ui-toc-dropdown .nav>.active:focus>a,.ui-toc-dropdown .nav>.active:hover>a,.ui-toc-dropdown .nav>.active>a{padding-left:18px;font-weight:700;color:#000;background-color:transparent;border-left:2px solid #000}.ui-toc-dropdown[dir=rtl] .nav>.active:focus>a,.ui-toc-dropdown[dir=rtl] .nav>.active:hover>a,.ui-toc-dropdown[dir=rtl] .nav>.active>a{padding-right:18px;border-left:none;border-right:2px solid #000}.ui-toc-dropdown .nav .nav{display:none;padding-bottom:10px}.ui-toc-dropdown .nav>.active>ul{display:block}.ui-toc-dropdown .nav .nav>li>a{padding-top:1px;padding-bottom:1px;padding-left:30px;font-size:12px;font-weight:400}.ui-toc-dropdown[dir=rtl] .nav .nav>li>a{padding-right:30px}.ui-toc-dropdown .nav .nav>li>ul>li>a{padding-top:1px;padding-bottom:1px;padding-left:40px;font-size:12px;font-weight:400}.ui-toc-dropdown[dir=rtl] .nav .nav>li>ul>li>a{padding-right:40px}.ui-toc-dropdown .nav .nav>li>a:focus,.ui-toc-dropdown .nav .nav>li>a:hover{padding-left:29px}.ui-toc-dropdown[dir=rtl] .nav .nav>li>a:focus,.ui-toc-dropdown[dir=rtl] .nav .nav>li>a:hover{padding-right:29px}.ui-toc-dropdown .nav .nav>li>ul>li>a:focus,.ui-toc-dropdown .nav .nav>li>ul>li>a:hover{padding-left:39px}.ui-toc-dropdown[dir=rtl] .nav .nav>li>ul>li>a:focus,.ui-toc-dropdown[dir=rtl] .nav .nav>li>ul>li>a:hover{padding-right:39px}.ui-toc-dropdown .nav .nav>.active:focus>a,.ui-toc-dropdown .nav .nav>.active:hover>a,.ui-toc-dropdown .nav .nav>.active>a{padding-left:28px;font-weight:500}.ui-toc-dropdown[dir=rtl] .nav .nav>.active:focus>a,.ui-toc-dropdown[dir=rtl] .nav .nav>.active:hover>a,.ui-toc-dropdown[dir=rtl] .nav .nav>.active>a{padding-right:28px}.ui-toc-dropdown .nav .nav>.active>.nav>.active:focus>a,.ui-toc-dropdown .nav .nav>.active>.nav>.active:hover>a,.ui-toc-dropdown .nav .nav>.active>.nav>.active>a{padding-left:38px;font-weight:500}.ui-toc-dropdown[dir=rtl] .nav .nav>.active>.nav>.active:focus>a,.ui-toc-dropdown[dir=rtl] .nav .nav>.active>.nav>.active:hover>a,.ui-toc-dropdown[dir=rtl] .nav .nav>.active>.nav>.active>a{padding-right:38px}.markdown-body{font-family:-apple-system,BlinkMacSystemFont,Segoe UI,Helvetica Neue,Helvetica,Roboto,Arial,sans-serif}html[lang^=ja] .markdown-body{font-family:-apple-system,BlinkMacSystemFont,Segoe UI,Helvetica Neue,Helvetica,Roboto,Arial,Hiragino Kaku Gothic Pro,ヒラギノ角ゴ Pro W3,Osaka,Meiryo,メイリオ,MS Gothic,ＭＳ\ ゴシック,sans-serif}html[lang=zh-tw] .markdown-body{font-family:-apple-system,BlinkMacSystemFont,Segoe UI,Helvetica Neue,Helvetica,Roboto,Arial,PingFang TC,Microsoft JhengHei,微軟正黑,sans-serif}html[lang=zh-cn] .markdown-body{font-family:-apple-system,BlinkMacSystemFont,Segoe UI,Helvetica Neue,Helvetica,Roboto,Arial,PingFang SC,Microsoft YaHei,微软雅黑,sans-serif}html .markdown-body[lang^=ja]{font-family:-apple-system,BlinkMacSystemFont,Segoe UI,Helvetica Neue,Helvetica,Roboto,Arial,Hiragino Kaku Gothic Pro,ヒラギノ角ゴ Pro W3,Osaka,Meiryo,メイリオ,MS Gothic,ＭＳ\ ゴシック,sans-serif}html .markdown-body[lang=zh-tw]{font-family:-apple-system,BlinkMacSystemFont,Segoe UI,Helvetica Neue,Helvetica,Roboto,Arial,PingFang TC,Microsoft JhengHei,微軟正黑,sans-serif}html .markdown-body[lang=zh-cn]{font-family:-apple-system,BlinkMacSystemFont,Segoe UI,Helvetica Neue,Helvetica,Roboto,Arial,PingFang SC,Microsoft YaHei,微软雅黑,sans-serif}html[lang^=ja] .ui-toc-dropdown{font-family:Source Sans Pro,Helvetica,Arial,Meiryo UI,MS PGothic,ＭＳ\ Ｐゴシック,sans-serif}html[lang=zh-tw] .ui-toc-dropdown{font-family:Source Sans Pro,Helvetica,Arial,Microsoft JhengHei UI,微軟正黑UI,sans-serif}html[lang=zh-cn] .ui-toc-dropdown{font-family:Source Sans Pro,Helvetica,Arial,Microsoft YaHei UI,微软雅黑UI,sans-serif}html .ui-toc-dropdown[lang^=ja]{font-family:Source Sans Pro,Helvetica,Arial,Meiryo UI,MS PGothic,ＭＳ\ Ｐゴシック,sans-serif}html .ui-toc-dropdown[lang=zh-tw]{font-family:Source Sans Pro,Helvetica,Arial,Microsoft JhengHei UI,微軟正黑UI,sans-serif}html .ui-toc-dropdown[lang=zh-cn]{font-family:Source Sans Pro,Helvetica,Arial,Microsoft YaHei UI,微软雅黑UI,sans-serif}.ui-affix-toc{position:fixed;top:0;max-width:15vw;max-height:70vh;overflow:auto}.back-to-top,.expand-toggle,.go-to-bottom{display:block;padding:4px 10px;margin-top:10px;margin-left:10px;font-size:12px;font-weight:500;color:#999}.back-to-top:focus,.back-to-top:hover,.expand-toggle:focus,.expand-toggle:hover,.go-to-bottom:focus,.go-to-bottom:hover{color:#563d7c;text-decoration:none}.back-to-top,.go-to-bottom{margin-top:0}.ui-user-icon{width:20px;height:20px;display:block;border-radius:3px;margin-top:2px;margin-bottom:2px;margin-right:5px;background-position:50%;background-repeat:no-repeat;background-size:cover}.ui-user-icon.small{width:18px;height:18px;display:inline-block;vertical-align:middle;margin:0 0 .2em}.ui-infobar>small>span{line-height:22px}.ui-infobar>small .dropdown{display:inline-block}.ui-infobar>small .dropdown a:focus,.ui-infobar>small .dropdown a:hover{text-decoration:none}.ui-published-note{color:#337ab7}.ui-published-note .fa{font-size:20px;vertical-align:top}.unselectable{-webkit-user-select:none;-o-user-select:none;user-select:none}@media print{blockquote,div,img,pre,table{page-break-inside:avoid!important}a[href]:after{font-size:12px!important}}.markdown-body.slides{position:relative;z-index:1;color:#222}.markdown-body.slides:before{content:"";display:block;position:absolute;top:0;left:0;right:0;bottom:0;z-index:-1;background-color:currentColor;box-shadow:0 0 0 50vw}.markdown-body.slides section[data-markdown]{position:relative;margin-bottom:1.5em;background-color:#fff;text-align:center}.markdown-body.slides section[data-markdown] code{text-align:left}.markdown-body.slides section[data-markdown]:before{content:"";display:block;padding-bottom:56.23%}.markdown-body.slides section[data-markdown]>div:first-child{position:absolute;top:50%;left:1em;right:1em;transform:translateY(-50%);max-height:100%;overflow:hidden}.markdown-body.slides section[data-markdown]>ul{display:inline-block}.markdown-body.slides>section>section+section:after{content:"";position:absolute;top:-1.5em;right:1em;height:1.5em;border:3px solid #777}body{font-smoothing:subpixel-antialiased!important;-webkit-font-smoothing:subpixel-antialiased!important;-moz-osx-font-smoothing:auto!important;text-shadow:0 0 1em transparent,1px 1px 1.2px rgba(0,0,0,.004);-webkit-overflow-scrolling:touch;letter-spacing:.025em}.focus,:focus{outline:none!important}::-moz-focus-inner{border:0!important}body{font-family:Source Sans Pro,Helvetica,Arial,sans-serif}html[lang^=ja] body{font-family:Source Sans Pro,Helvetica,Arial,Hiragino Kaku Gothic Pro,ヒラギノ角ゴ Pro W3,Osaka,Meiryo,メイリオ,MS Gothic,ＭＳ\ ゴシック,sans-serif}html[lang=zh-tw] body{font-family:Source Sans Pro,Helvetica,Arial,PingFang TC,Microsoft JhengHei,微軟正黑,sans-serif}html[lang=zh-cn] body{font-family:Source Sans Pro,Helvetica,Arial,PingFang SC,Microsoft YaHei,微软雅黑,sans-serif}abbr[title]{border-bottom:none;text-decoration:underline;-webkit-text-decoration:underline dotted;text-decoration:underline dotted}abbr[data-original-title],abbr[title]{cursor:help}body.modal-open{overflow-y:auto;padding-right:0!important}
    </style>
    <!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries -->
    <!-- WARNING: Respond.js doesn't work if you view the page via file:// -->
    <!--[if lt IE 9]>
    	<script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js" integrity="sha256-3Jy/GbSLrg0o9y5Z5n1uw0qxZECH7C6OQpVBgNFYa0g=" crossorigin="anonymous"></script>
    	<script src="https://cdnjs.cloudflare.com/ajax/libs/respond.js/1.4.2/respond.min.js" integrity="sha256-g6iAfvZp+nDQ2TdTR/VVKJf3bGro4ub5fvWSWVRi2NE=" crossorigin="anonymous"></script>
		<script src="https://cdnjs.cloudflare.com/ajax/libs/es5-shim/4.5.9/es5-shim.min.js" integrity="sha256-8E4Is26QH0bD52WoQpcB+R/tcWQtpzlCojrybUd7Mxo=" crossorigin="anonymous"></script>
    <![endif]-->
</head>

<body>
    <div id="doc" class="markdown-body container-fluid comment-enabled" data-hard-breaks="true" style="position: relative;"><h1 id="Implementation-of-ResNet"><a class="anchor hidden-xs" href="#Implementation-of-ResNet" title="Implementation-of-ResNet"><span class="octicon octicon-link"></span></a>Implementation of ResNet</h1><p><br><span style="font-size:14pt;">We will use the tensorflow.keras Functional API to build ResNet</span><br>
(<a href="https://arxiv.org/pdf/1512.03385.pdf" target="_blank" rel="noopener">https://arxiv.org/pdf/1512.03385.pdf</a>)</p><hr><p>In the paper we can read:</p><blockquote>
<p><strong>[i]</strong> “We adopt batch normalization (BN) [16] right after each convolution and before activation.”<br>
<strong>[ii]</strong> “Donwsampling is performed by conv3_1, conv4_1, and conv5_1 with a stride of 2.”<br>
<strong>[iii]</strong> “(B) The projection shortcut in Eqn.(2) is used to match dimensions (done by 1×1 convolutions). For both options, when the shortcuts go across feature maps of two sizes, they are performed with a stride of 2”<br>
<strong>[iv]</strong> “[…] (B) projection shortcuts are used for increasing dimensions, and other shortcuts are identity;”<br>
<strong>[v]</strong> “The three layers are 1×1, 3×3, and 1×1 convolutions, where the 1×1 layers are responsible for reducing and then increasing (restoring) dimensions, leaving the 3×3 layer a bottleneck with smaller input/output dimensions.”<br>
<strong>[vi]</strong> “50-layer ResNet: We replace each 2-layer block in the 34-layer net with this 3-layer bottleneck block, resulting in a 50-layer ResNet (Table 1). We use option B for increasing<br>
dimensions.”</p>
</blockquote><br><p>We will also make use of the following Table <strong>[vii]</strong>:</p><img src="https://github.com/Machine-Learning-Tokyo/DL-workshop-series/raw/master/Part%20I%20-%20Convolution%20Operations/images/ResNet/ResNet.png" width="600"><br><br><p>as well the following diagram <strong>[viii]</strong>:<br>
<img src="https://github.com/Machine-Learning-Tokyo/DL-workshop-series/raw/master/Part%20I%20-%20Convolution%20Operations/images/ResNet/ResNet_block.png" width="200"></p><hr><h2 id="Network-architecture"><a class="anchor hidden-xs" href="#Network-architecture" title="Network-architecture"><span class="octicon octicon-link"></span></a>Network architecture</h2><p>The network starts with a [Conv, BatchNorm, ReLU] block (<strong>[i]</strong>) and continues with a series of <strong>Resnet blocks</strong> (conv<em>n</em>.x in <strong>[vii]</strong>) before the final <em>Avg Pool</em> and <em>Fully Connected</em> layers.</p><h3 id="Resnet-block"><a class="anchor hidden-xs" href="#Resnet-block" title="Resnet-block"><span class="octicon octicon-link"></span></a>Resnet block</h3><p>The <em>Resnet block</em> consists of a repetition of blocks similar to the one depicted in <strong>[viii]</strong>. As one can see the input tesnor goes through three Conv-BN-ReLU blocks and the output is added to the input tensor. This type of connection that skips the main body of the block and merges (adds) the input tensor with another one further on is called <em>skip connection</em> (right arrow in <strong>[viii]</strong>).</p><p>There are two types of skip connections in ResNet: the <strong>Identity</strong> and the <strong>Projection</strong>. In <strong>[viii]</strong> is depicted the <strong>Identity</strong> one. This is used when the input tensor has same shape as the one produced by the last Convolution layer of the block.</p><p>However, when the two tensors have different shape, the input tensor must change to get same shape as the other one in order to be able to be added to it. This is done by the <strong>Projection</strong> connection as described in <strong>[iii]</strong> and <strong>[iv]</strong>.</p><p>The change in shape happens when we:</p><ul>
<li>Change the number of filters and thus of feature maps of the output tensor.<br>
This happens at the first sub-block of each <em>ResNet</em> block since the output tensor has 4 times the number of feature maps than the input tensor.</li>
<li>Change the spatial dimensions of the output tensor (downsampling)<br>
which takes place according to <strong>[ii]</strong>.</li>
</ul><h4 id="Identity-block"><a class="anchor hidden-xs" href="#Identity-block" title="Identity-block"><span class="octicon octicon-link"></span></a>Identity block</h4><p>The <em>Identity block</em> takes a tensor as an input and passes it through 1 stream of:</p><blockquote>
<ol>
<li>a 1x1 <em>Convolution</em> layer followed by a <em>Batch Normalization</em> and a <em>Rectified Linear Unit (ReLU)</em> activation layer</li>
<li>a 3x3 <em>Convolution</em> layer followed by a <em>Batch Normalization</em> and a <em>Rectified Linear Unit (ReLU)</em> activation layer</li>
<li>a 1x1 <em>Convolution</em> layer followed by a <em>Batch Normalization</em> layer</li>
</ol>
<p>Pay attention at the number of filters (depicted with the letter f at the diagram) which are the same for the first 2 Convolution layer but 4x for the 3rd one.</p>
</blockquote><p>Then the <em>output</em> of this stream is added to the <em>input</em> tensor. On the new tensor a <em>Rectified Linear Unit (ReLU)</em> activation is applied befor returning it.</p><br><h4 id="Projection-block"><a class="anchor hidden-xs" href="#Projection-block" title="Projection-block"><span class="octicon octicon-link"></span></a>Projection block</h4><p>The <em>Projection block</em> takes a tensor as an input and passes it through 2 streams.</p><ul>
<li>The left stream consists of:</li>
</ul><blockquote>
<ol>
<li>a 1x1 <em>Convolution</em> layer followed by a <em>Batch Normalization</em> and a <em>Rectified Linear Unit (ReLU)</em> activation layer</li>
<li>a 3x3 <em>Convolution</em> layer followed by a <em>Batch Normalization</em> and a <em>Rectified Linear Unit (ReLU)</em> activation layer</li>
<li>a 1x1 <em>Convolution</em> layer followed by a <em>Batch Normalization</em> layer</li>
</ol>
<p>Pay attention at the number of filters (depicted with the letter f at the diagram) which are the same for the first 2 Convolution layer but 4x for the 3rd one.</p>
</blockquote><ul>
<li>The right stream consists of:</li>
</ul><blockquote>
<p>a 1x1 <em>Convolution</em> layer followed by a <em>Batch Normalization</em> layer</p>
</blockquote><p>The outputs of both streams are then added up to a new tensor on which a <em>Rectified Linear Unit (ReLU)</em> activation is applied befor returning it.</p><br><p>As one can see the only difference between the two blocks is the existence of the Convolution-Batch Normalization sub-block at the right stream.</p><p>The reason we need this Convolution layer is:</p><ul>
<li>To change the number of filters (feature maps) of the tensor after each block.</li>
<li>To change the size of the tensor after each block.</li>
</ul><p>In order to change the size (downsampling) we use a stride of 2 after specific blocks as described at <strong>[ii]</strong> at the first 1x1 Convolution layer and the Projection’s Convolution layer according to <strong>[iii]</strong> and <strong>[v]</strong>.</p><hr><h2 id="Workflow"><a class="anchor hidden-xs" href="#Workflow" title="Workflow"><span class="octicon octicon-link"></span></a>Workflow</h2><p>We will:</p><ol>
<li>import the neccesary layers</li>
<li>write a helper function for the Conv-BatchNorm-ReLU block (<strong>[i]</strong>)</li>
<li>write a helper function for the Identity block</li>
<li>write a helper function for the Projection block</li>
<li>write a helper function for the Resnet block (<strong>[ii]</strong>)</li>
<li>use these helper functions to build the model.</li>
</ol><hr><h3 id="1-Imports"><a class="anchor hidden-xs" href="#1-Imports" title="1-Imports"><span class="octicon octicon-link"></span></a>1. Imports</h3><p><strong>Code:</strong></p><blockquote>
<pre><code class="python hljs"><span class="hljs-keyword">from</span> tensorflow.keras.layers <span class="hljs-keyword">import</span> Input, Conv2D, BatchNormalization, \
     ReLU, Add, MaxPool2D, GlobalAvgPool2D, Dense
</code></pre>
</blockquote><hr><h3 id="2-Conv-BatchNorm-ReLU-block"><a class="anchor hidden-xs" href="#2-Conv-BatchNorm-ReLU-block" title="2-Conv-BatchNorm-ReLU-block"><span class="octicon octicon-link"></span></a>2. <em>Conv-BatchNorm-ReLU block</em></h3><p>Next, we will build the <em>Conv-BatchNorm-ReLU block</em> as a function that will:</p><ul>
<li>take as inputs:
<ul>
<li>a tensor (<strong><code>x</code></strong>)</li>
<li>the number of filters (<strong><code>filters</code></strong>)</li>
<li>the kernel size (<strong><code>kernel_size</code></strong>)</li>
<li>the strides (<strong><code>strides</code></strong>)</li>
</ul>
</li>
<li>run:
<ul>
<li>apply a <em>Convolution layer</em> followed by a <em>Batch Normalization</em> and a <em>ReLU</em> activation</li>
</ul>
</li>
<li>return the tensor</li>
</ul><p><strong>Code:</strong></p><blockquote>
<pre><code class="python hljs"><span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">conv_batchnorm_relu</span><span class="hljs-params">(x, filters, kernel_size, strides)</span>:</span>
    x = Conv2D(filters=filters,
               kernel_size=kernel_size,
               strides=strides,
               padding=<span class="hljs-string">'same'</span>)(x)
    x = BatchNormalization()(x)
    x = ReLU()(x)
    <span class="hljs-keyword">return</span> x
</code></pre>
</blockquote><hr><h3 id="3-Identity-block"><a class="anchor hidden-xs" href="#3-Identity-block" title="3-Identity-block"><span class="octicon octicon-link"></span></a>3. <em>Identity block</em></h3><p>Now, we will build the <em>Identity block</em> as a function that will:</p><ul>
<li>take as inputs:
<ul>
<li>a tensor (<strong><code>tensor</code></strong>)</li>
<li>the number of filters (<strong><code>filters</code></strong>)</li>
<li>the kernel size (<strong><code>kernel_size</code></strong>)</li>
<li>the strides (<strong><code>strides</code></strong>)</li>
</ul>
</li>
<li>run:
<ul>
<li>apply a 1x1 <strong>Conv-BatchNorm-ReLU block</strong> to <strong><code>tensor</code></strong></li>
<li>apply a 3x3 <strong>Conv-BatchNorm-ReLU block</strong></li>
<li>apply a 1x1 <em>Convolution layer</em> with 4 times the filters <strong><code>filters</code></strong></li>
<li>apply a <em>Batch normalization</em></li>
<li>add this tensor with <strong><code>tensor</code></strong></li>
<li>apply a <em>ReLU</em> activation</li>
</ul>
</li>
<li>return the tensor</li>
</ul><p><strong>Code:</strong></p><blockquote>
<pre><code class="python hljs"><span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">identity_block</span><span class="hljs-params">(tensor, filters)</span>:</span>
    x = conv_batchnorm_relu(tensor, filters=filters, kernel_size=<span class="hljs-number">1</span>, strides=<span class="hljs-number">1</span>)
    x = conv_batchnorm_relu(x, filters=filters, kernel_size=<span class="hljs-number">3</span>, strides=<span class="hljs-number">1</span>)
    x = Conv2D(filters=<span class="hljs-number">4</span>*filters, kernel_size=<span class="hljs-number">1</span>, strides=<span class="hljs-number">1</span>)(x)  <span class="hljs-comment"># notice: filters=4*filters</span>
    x = BatchNormalization()(x)

    x = Add()([x, tensor])
    x = ReLU()(x)
    <span class="hljs-keyword">return</span> x
</code></pre>
</blockquote><hr><h3 id="4-Projection-block"><a class="anchor hidden-xs" href="#4-Projection-block" title="4-Projection-block"><span class="octicon octicon-link"></span></a>4. <em>Projection block</em></h3><p>Now, we will build the <em>Projection block</em> which is similar to the <em>Identity</em> one.</p><p>Remember, this time we need the strides because we want to downsample the tensors at specific blocks according to <strong>[ii]</strong>, <strong>[iii]</strong> and <strong>[v]</strong>:</p><blockquote>
<p>“the 1×1 layers are responsible for reducing and then increasing (restoring) dimensions”.</p>
</blockquote><p>The downsampling at the main stream will take place at the first 1x1 Convolution layer*.<br>
The downsampling at the right stream will take place at its Convolution layer.</p><p><strong>Code:</strong></p><blockquote>
<pre><code class="python hljs"><span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">projection_block</span><span class="hljs-params">(tensor, filters, strides)</span>:</span>
    <span class="hljs-comment"># left stream</span>
    x = conv_batchnorm_relu(tensor, filters=filters, kernel_size=<span class="hljs-number">1</span>, strides=strides) <span class="hljs-comment">#[v]</span>
    x = conv_batchnorm_relu(x, filters=filters, kernel_size=<span class="hljs-number">3</span>, strides=<span class="hljs-number">1</span>)
    x = Conv2D(filters=<span class="hljs-number">4</span>*filters, kernel_size=<span class="hljs-number">1</span>, strides=<span class="hljs-number">1</span>)(x)  <span class="hljs-comment"># notice: filters=4*filters</span>
    x = BatchNormalization()(x)

    <span class="hljs-comment"># right stream</span>
    shortcut = Conv2D(filters=<span class="hljs-number">4</span>*filters, kernel_size=<span class="hljs-number">1</span>, strides=strides)(tensor)  <span class="hljs-comment"># notice: filters=4*filters</span>
    shortcut = BatchNormalization()(shortcut)

    x = Add()([x, shortcut])
    x = ReLU()(x)
    <span class="hljs-keyword">return</span> x
</code></pre>
</blockquote><p>*<em>Notice that in some implementations downsampling takes place at the 3x3 layer. This is also know as ResNet 1.5 (<a href="https://ngc.nvidia.com/catalog/model-scripts/nvidia:resnet_50_v1_5_for_pytorch" target="_blank" rel="noopener">https://ngc.nvidia.com/catalog/model-scripts/nvidia:resnet_50_v1_5_for_pytorch</a>).</em></p><hr><h3 id="5-Resnet-block"><a class="anchor hidden-xs" href="#5-Resnet-block" title="5-Resnet-block"><span class="octicon octicon-link"></span></a>5. <em>Resnet block</em></h3><p>Now that we defined the <em>Projection block</em> and the <em>Identity block</em> we can use them to define the <strong>Resnet block</strong>.</p><p>Based on the <strong>[vii]</strong> (column <em>50-layer</em>) for each block we have a number of repetiontions (depicted with <em>xn</em> next to the block numbers). The 1st of these blocks will be a <em>Projection block</em> and the rest will be <em>Identity blocks</em>.</p><p>The reason for this is that at the beginning of each block the number of feature maps of the tensor change. Since at the Identity block the input tensor and the output tensor are added, they need to have the same number of feature maps.</p><p>Let’s build the <em>Resnet block</em> as a function that will:</p><ul>
<li>take as inputs:
<ul>
<li>a tensor (<strong><code>x</code></strong>)</li>
<li>the number of filters (<strong><code>filters</code></strong>)</li>
<li>the total number of repetitions of internal blocks (<strong><code>reps</code></strong>)</li>
<li>the strides (<strong><code>strides</code></strong>)</li>
</ul>
</li>
<li>run:
<ul>
<li>apply a projection block with strides: <strong><code>strides</code></strong></li>
<li>for apply an <em>Identity block</em> for <span class="mathjax"><span class="MathJax_Preview" style="color: inherit;"></span><span id="MathJax-Element-1-Frame" class="mjx-chtml MathJax_CHTML" tabindex="0" data-mathml="<math xmlns=&quot;http://www.w3.org/1998/Math/MathML&quot;><mi>r</mi><mo>&amp;#x2212;</mo><mn>1</mn></math>" role="presentation" style="font-size: 118%; position: relative;"><span id="MJXc-Node-1" class="mjx-math" aria-hidden="true"><span id="MJXc-Node-2" class="mjx-mrow"><span id="MJXc-Node-3" class="mjx-mi"><span class="mjx-char MJXc-TeX-math-I" style="padding-top: 0.215em; padding-bottom: 0.267em;">r</span></span><span id="MJXc-Node-4" class="mjx-mo MJXc-space2"><span class="mjx-char MJXc-TeX-main-R" style="padding-top: 0.32em; padding-bottom: 0.426em;">−</span></span><span id="MJXc-Node-5" class="mjx-mn MJXc-space2"><span class="mjx-char MJXc-TeX-main-R" style="padding-top: 0.373em; padding-bottom: 0.32em;">1</span></span></span></span><span class="MJX_Assistive_MathML" role="presentation"><math xmlns="http://www.w3.org/1998/Math/MathML"><mi>r</mi><mo>−</mo><mn>1</mn></math></span></span><script type="math/tex" id="MathJax-Element-1">r-1</script></span> times (the <span class="mathjax"><span class="MathJax_Preview" style="color: inherit;"></span><span id="MathJax-Element-2-Frame" class="mjx-chtml MathJax_CHTML" tabindex="0" data-mathml="<math xmlns=&quot;http://www.w3.org/1998/Math/MathML&quot;><mo>&amp;#x2212;</mo><mn>1</mn></math>" role="presentation" style="font-size: 118%; position: relative;"><span id="MJXc-Node-6" class="mjx-math" aria-hidden="true"><span id="MJXc-Node-7" class="mjx-mrow"><span id="MJXc-Node-8" class="mjx-mo"><span class="mjx-char MJXc-TeX-main-R" style="padding-top: 0.32em; padding-bottom: 0.426em;">−</span></span><span id="MJXc-Node-9" class="mjx-mn"><span class="mjx-char MJXc-TeX-main-R" style="padding-top: 0.373em; padding-bottom: 0.32em;">1</span></span></span></span><span class="MJX_Assistive_MathML" role="presentation"><math xmlns="http://www.w3.org/1998/Math/MathML"><mo>−</mo><mn>1</mn></math></span></span><script type="math/tex" id="MathJax-Element-2">-1</script></span> is because the first block was a <em>Convolution</em> one)</li>
</ul>
</li>
<li>return the tensor</li>
</ul><p><strong>Code:</strong></p><blockquote>
<pre><code class="python hljs"><span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">resnet_block</span><span class="hljs-params">(x, filters, reps, strides)</span>:</span>
    x = projection_block(x, filters=filters, strides=strides)
    <span class="hljs-keyword">for</span> _ <span class="hljs-keyword">in</span> range(reps<span class="hljs-number">-1</span>):
        x = identity_block(x, filters=filters)
    <span class="hljs-keyword">return</span> x
</code></pre>
</blockquote><hr><h3 id="6-Model-code"><a class="anchor hidden-xs" href="#6-Model-code" title="6-Model-code"><span class="octicon octicon-link"></span></a>6. Model code</h3><p>Now we are ready to build the model:</p><p><strong>Code:</strong></p><blockquote>
<pre><code class="python hljs">input = Input(shape=(<span class="hljs-number">224</span>, <span class="hljs-number">224</span>, <span class="hljs-number">3</span>))

x = conv_batchnorm_relu(input, filters=<span class="hljs-number">64</span>, kernel_size=<span class="hljs-number">7</span>, strides=<span class="hljs-number">2</span>)  <span class="hljs-comment"># [3]: 7x7, 64, strides 2</span>
x = MaxPool2D(pool_size=<span class="hljs-number">3</span>, strides=<span class="hljs-number">2</span>, padding=<span class="hljs-string">'same'</span>)(x)  <span class="hljs-comment"># [3]: 3x3 max mool, strides 2</span>

x = resnet_block(x, filters=<span class="hljs-number">64</span>, reps=<span class="hljs-number">3</span>, strides=<span class="hljs-number">1</span>)
x = resnet_block(x, filters=<span class="hljs-number">128</span>, reps=<span class="hljs-number">4</span>, strides=<span class="hljs-number">2</span>)  <span class="hljs-comment"># strides=2 ([2]: conv3_1)</span>
x = resnet_block(x, filters=<span class="hljs-number">256</span>, reps=<span class="hljs-number">6</span>, strides=<span class="hljs-number">2</span>)  <span class="hljs-comment"># strides=2 ([2]: conv4_1)</span>
x = resnet_block(x, filters=<span class="hljs-number">512</span>, reps=<span class="hljs-number">3</span>, strides=<span class="hljs-number">2</span>)  <span class="hljs-comment"># strides=2 ([2]: conv5_1)</span>

x = GlobalAvgPool2D()(x)  <span class="hljs-comment"># [3]: average pool *it is not written any pool size so we use Global</span>

output = Dense(<span class="hljs-number">1000</span>, activation=<span class="hljs-string">'softmax'</span>)(x)  <span class="hljs-comment"># [3]: 1000-d fc, softmax</span>

<span class="hljs-keyword">from</span> tensorflow.keras <span class="hljs-keyword">import</span> Model

model = Model(input, output)
</code></pre>
</blockquote><hr><h2 id="Final-code"><a class="anchor hidden-xs" href="#Final-code" title="Final-code"><span class="octicon octicon-link"></span></a>Final code</h2><p><strong>Code:</strong></p><pre><code class="python hljs"><span class="hljs-keyword">from</span> tensorflow.keras.layers <span class="hljs-keyword">import</span> Input
<span class="hljs-keyword">from</span> tensorflow.keras.layers <span class="hljs-keyword">import</span> Conv2D, BatchNormalization, ReLU, Add
<span class="hljs-keyword">from</span> tensorflow.keras.layers <span class="hljs-keyword">import</span> MaxPool2D, GlobalAvgPool2D, Dense


<span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">conv_batchnorm_relu</span><span class="hljs-params">(x, filters, kernel_size, strides)</span>:</span>
    x = Conv2D(filters=filters,
               kernel_size=kernel_size,
               strides=strides,
               padding=<span class="hljs-string">'same'</span>)(x)
    x = BatchNormalization()(x)
    x = ReLU()(x)
    <span class="hljs-keyword">return</span> x


<span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">identity_block</span><span class="hljs-params">(tensor, filters)</span>:</span>
    x = conv_batchnorm_relu(tensor, filters=filters, kernel_size=<span class="hljs-number">1</span>, strides=<span class="hljs-number">1</span>)
    x = conv_batchnorm_relu(x, filters=filters, kernel_size=<span class="hljs-number">3</span>, strides=<span class="hljs-number">1</span>)
    x = Conv2D(filters=<span class="hljs-number">4</span>*filters, kernel_size=<span class="hljs-number">1</span>, strides=<span class="hljs-number">1</span>)(x)  <span class="hljs-comment"># notice: filters=4*filters</span>
    x = BatchNormalization()(x)

    x = Add()([x, tensor])
    x = ReLU()(x)
    <span class="hljs-keyword">return</span> x


<span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">projection_block</span><span class="hljs-params">(tensor, filters, strides)</span>:</span>
    <span class="hljs-comment"># left stream</span>
    x = conv_batchnorm_relu(tensor, filters=filters, kernel_size=<span class="hljs-number">1</span>, strides=strides)
    x = conv_batchnorm_relu(x, filters=filters, kernel_size=<span class="hljs-number">3</span>, strides=<span class="hljs-number">1</span>)
    x = Conv2D(filters=<span class="hljs-number">4</span>*filters, kernel_size=<span class="hljs-number">1</span>, strides=<span class="hljs-number">1</span>)(x)  <span class="hljs-comment"># notice: filters=4*filters</span>
    x = BatchNormalization()(x)

    <span class="hljs-comment"># right stream</span>
    shortcut = Conv2D(filters=<span class="hljs-number">4</span>*filters, kernel_size=<span class="hljs-number">1</span>, strides=strides)(tensor)  <span class="hljs-comment"># notice: filters=4*filters</span>
    shortcut = BatchNormalization()(shortcut)

    x = Add()([x, shortcut])
    x = ReLU()(x)
    <span class="hljs-keyword">return</span> x


<span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">resnet_block</span><span class="hljs-params">(x, filters, reps, strides)</span>:</span>
    x = projection_block(x, filters=filters, strides=strides)
    <span class="hljs-keyword">for</span> _ <span class="hljs-keyword">in</span> range(reps<span class="hljs-number">-1</span>):  <span class="hljs-comment"># the -1 is because the first block was a Conv one</span>
        x = identity_block(x, filters=filters)
    <span class="hljs-keyword">return</span> x


input = Input(shape=(<span class="hljs-number">224</span>, <span class="hljs-number">224</span>, <span class="hljs-number">3</span>))

x = conv_batchnorm_relu(input, filters=<span class="hljs-number">64</span>, kernel_size=<span class="hljs-number">7</span>, strides=<span class="hljs-number">2</span>)  <span class="hljs-comment"># [3]: 7x7, 64, strides 2</span>
x = MaxPool2D(pool_size=<span class="hljs-number">3</span>, strides=<span class="hljs-number">2</span>, padding=<span class="hljs-string">'same'</span>)(x)  <span class="hljs-comment"># [3]: 3x3 max mool, strides 2</span>

x = resnet_block(x, filters=<span class="hljs-number">64</span>, reps=<span class="hljs-number">3</span>, strides=<span class="hljs-number">1</span>)
x = resnet_block(x, filters=<span class="hljs-number">128</span>, reps=<span class="hljs-number">4</span>, strides=<span class="hljs-number">2</span>)  <span class="hljs-comment"># s=2 ([2]: conv3_1)</span>
x = resnet_block(x, filters=<span class="hljs-number">256</span>, reps=<span class="hljs-number">6</span>, strides=<span class="hljs-number">2</span>)  <span class="hljs-comment"># s=2 ([2]: conv4_1)</span>
x = resnet_block(x, filters=<span class="hljs-number">512</span>, reps=<span class="hljs-number">3</span>, strides=<span class="hljs-number">2</span>)  <span class="hljs-comment"># s=2 ([2]: conv5_1)</span>

x = GlobalAvgPool2D()(x)  <span class="hljs-comment"># [3]: average pool *it is not written any pool size so we use Global</span>

output = Dense(<span class="hljs-number">1000</span>, activation=<span class="hljs-string">'softmax'</span>)(x)  <span class="hljs-comment"># [3]: 1000-d fc, softmax</span>

<span class="hljs-keyword">from</span> tensorflow.keras <span class="hljs-keyword">import</span> Model

model = Model(input, output)
</code></pre><hr><h2 id="Model-diagram"><a class="anchor hidden-xs" href="#Model-diagram" title="Model-diagram"><span class="octicon octicon-link"></span></a>Model diagram</h2><img src="https://raw.githubusercontent.com/Machine-Learning-Tokyo/CNN-Architectures/master/Implementations/ResNet/ResNet_diagram.svg?sanitize=true"></div>
    <div class="ui-toc dropup unselectable hidden-print" style="display:none;">
        <div class="pull-right dropdown">
            <a id="tocLabel" class="ui-toc-label btn btn-default" data-toggle="dropdown" href="#" role="button" aria-haspopup="true" aria-expanded="false" title="Table of content">
                <i class="fa fa-bars"></i>
            </a>
            <ul id="ui-toc" class="ui-toc-dropdown dropdown-menu" aria-labelledby="tocLabel">
                <div class="toc"><ul class="nav">
<li class=""><a href="#Implementation-of-ResNet" title="Implementation of ResNet">Implementation of ResNet</a><ul class="nav">
<li><a href="#Network-architecture" title="Network architecture">Network architecture</a><ul class="nav">
<li><a href="#Resnet-block" title="Resnet block">Resnet block</a></li>
</ul>
</li>
<li><a href="#Workflow" title="Workflow">Workflow</a><ul class="nav">
<li><a href="#1-Imports" title="1. Imports">1. Imports</a></li>
<li><a href="#2-Conv-BatchNorm-ReLU-block" title="2. Conv-BatchNorm-ReLU block">2. Conv-BatchNorm-ReLU block</a></li>
<li><a href="#3-Identity-block" title="3. Identity block">3. Identity block</a></li>
<li><a href="#4-Projection-block" title="4. Projection block">4. Projection block</a></li>
<li><a href="#5-Resnet-block" title="5. Resnet block">5. Resnet block</a></li>
<li><a href="#6-Model-code" title="6. Model code">6. Model code</a></li>
</ul>
</li>
<li class=""><a href="#Final-code" title="Final code">Final code</a></li>
<li><a href="#Model-diagram" title="Model diagram">Model diagram</a></li>
</ul>
</li>
</ul>
</div><div class="toc-menu"><a class="expand-toggle" href="#">Expand all</a><a class="back-to-top" href="#">Back to top</a><a class="go-to-bottom" href="#">Go to bottom</a></div>
            </ul>
        </div>
    </div>
    <div id="ui-toc-affix" class="ui-affix-toc ui-toc-dropdown unselectable hidden-print" data-spy="affix" style="top:17px;display:none;" null null>
        <div class="toc"><ul class="nav">
<li class=""><a href="#Implementation-of-ResNet" title="Implementation of ResNet">Implementation of ResNet</a><ul class="nav">
<li><a href="#Network-architecture" title="Network architecture">Network architecture</a><ul class="nav">
<li><a href="#Resnet-block" title="Resnet block">Resnet block</a></li>
</ul>
</li>
<li><a href="#Workflow" title="Workflow">Workflow</a><ul class="nav">
<li><a href="#1-Imports" title="1. Imports">1. Imports</a></li>
<li><a href="#2-Conv-BatchNorm-ReLU-block" title="2. Conv-BatchNorm-ReLU block">2. Conv-BatchNorm-ReLU block</a></li>
<li><a href="#3-Identity-block" title="3. Identity block">3. Identity block</a></li>
<li><a href="#4-Projection-block" title="4. Projection block">4. Projection block</a></li>
<li><a href="#5-Resnet-block" title="5. Resnet block">5. Resnet block</a></li>
<li><a href="#6-Model-code" title="6. Model code">6. Model code</a></li>
</ul>
</li>
<li class=""><a href="#Final-code" title="Final code">Final code</a></li>
<li><a href="#Model-diagram" title="Model diagram">Model diagram</a></li>
</ul>
</li>
</ul>
</div><div class="toc-menu"><a class="expand-toggle" href="#">Expand all</a><a class="back-to-top" href="#">Back to top</a><a class="go-to-bottom" href="#">Go to bottom</a></div>
    </div>
    <script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.1.1/jquery.min.js" integrity="sha256-hVVnYaiADRTO2PzUGmuLJr8BLUSjGIZsDYGmIJLv2b8=" crossorigin="anonymous"></script>
    <script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.3.7/js/bootstrap.min.js" integrity="sha256-U5ZEeKfGNOja007MMD3YBI0A3OSZOQbeG6z2f2Y0hu8=" crossorigin="anonymous" defer></script>
    <script src="https://cdnjs.cloudflare.com/ajax/libs/gist-embed/2.6.0/gist-embed.min.js" integrity="sha256-KyF2D6xPIJUW5sUDSs93vWyZm+1RzIpKCexxElmxl8g=" crossorigin="anonymous" defer></script>
    <script>
        var markdown = $(".markdown-body");
        //smooth all hash trigger scrolling
        function smoothHashScroll() {
            var hashElements = $("a[href^='#']").toArray();
            for (var i = 0; i < hashElements.length; i++) {
                var element = hashElements[i];
                var $element = $(element);
                var hash = element.hash;
                if (hash) {
                    $element.on('click', function (e) {
                        // store hash
                        var hash = this.hash;
                        if ($(hash).length <= 0) return;
                        // prevent default anchor click behavior
                        e.preventDefault();
                        // animate
                        $('body, html').stop(true, true).animate({
                            scrollTop: $(hash).offset().top
                        }, 100, "linear", function () {
                            // when done, add hash to url
                            // (default click behaviour)
                            window.location.hash = hash;
                        });
                    });
                }
            }
        }

        smoothHashScroll();
        var toc = $('.ui-toc');
        var tocAffix = $('.ui-affix-toc');
        var tocDropdown = $('.ui-toc-dropdown');
        //toc
        tocDropdown.click(function (e) {
            e.stopPropagation();
        });

        var enoughForAffixToc = true;

        function generateScrollspy() {
            $(document.body).scrollspy({
                target: ''
            });
            $(document.body).scrollspy('refresh');
            if (enoughForAffixToc) {
                toc.hide();
                tocAffix.show();
            } else {
                tocAffix.hide();
                toc.show();
            }
            $(document.body).scroll();
        }

        function windowResize() {
            //toc right
            var paddingRight = parseFloat(markdown.css('padding-right'));
            var right = ($(window).width() - (markdown.offset().left + markdown.outerWidth() - paddingRight));
            toc.css('right', right + 'px');
            //affix toc left
            var newbool;
            var rightMargin = (markdown.parent().outerWidth() - markdown.outerWidth()) / 2;
            //for ipad or wider device
            if (rightMargin >= 133) {
                newbool = true;
                var affixLeftMargin = (tocAffix.outerWidth() - tocAffix.width()) / 2;
                var left = markdown.offset().left + markdown.outerWidth() - affixLeftMargin;
                tocAffix.css('left', left + 'px');
            } else {
                newbool = false;
            }
            if (newbool != enoughForAffixToc) {
                enoughForAffixToc = newbool;
                generateScrollspy();
            }
        }
        $(window).resize(function () {
            windowResize();
        });
        $(document).ready(function () {
            windowResize();
            generateScrollspy();
        });

        //remove hash
        function removeHash() {
            window.location.hash = '';
        }

        var backtotop = $('.back-to-top');
        var gotobottom = $('.go-to-bottom');

        backtotop.click(function (e) {
            e.preventDefault();
            e.stopPropagation();
            if (scrollToTop)
                scrollToTop();
            removeHash();
        });
        gotobottom.click(function (e) {
            e.preventDefault();
            e.stopPropagation();
            if (scrollToBottom)
                scrollToBottom();
            removeHash();
        });

        var toggle = $('.expand-toggle');
        var tocExpand = false;

        checkExpandToggle();
        toggle.click(function (e) {
            e.preventDefault();
            e.stopPropagation();
            tocExpand = !tocExpand;
            checkExpandToggle();
        })

        function checkExpandToggle () {
            var toc = $('.ui-toc-dropdown .toc');
            var toggle = $('.expand-toggle');
            if (!tocExpand) {
                toc.removeClass('expand');
                toggle.text('Expand all');
            } else {
                toc.addClass('expand');
                toggle.text('Collapse all');
            }
        }

        function scrollToTop() {
            $('body, html').stop(true, true).animate({
                scrollTop: 0
            }, 100, "linear");
        }

        function scrollToBottom() {
            $('body, html').stop(true, true).animate({
                scrollTop: $(document.body)[0].scrollHeight
            }, 100, "linear");
        }
    </script>
</body>

</html>
